from __future__ import division
from operator import itemgetter, attrgetter

import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set

print "Program Begins..."
# What is the query length distribution for the fake one, it seems that the fake one is too fake for me
# key: queryLength in int format
# value: freq in int format
queryLengthDistributionDict = {}
# option1
# inputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/100KQueries_head_95K"
# option2
# inputFileName = "/data/jrodri04/lm/10M.95KQ.5gram.ModKN.txt"
# option3
# inputFileName = "/data/jrodri04/lm/30M.95KQ.ModKN.txt"
# option4
inputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/95KQueriesMachineGenerated3_20130818_using_new_fake_queryLog"

inputFileHandler = open(inputFileName,"r")
currentLine = inputFileHandler.readline()
lineCounter = 1
while currentLine:
    # option1
    # data = currentLine.strip().split(":")[1]
    # option2
    # data = currentLine.strip().lower()
    # option3
    data = currentLine.strip().split(":")[2]
    
    '''
    # processing option1 (the ORIGINAL one)
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    queryContent = data
    '''
    
    # processing option2 (the simple split one)
    queryContent = data
    
    
    queryContentElements = queryContent.strip().split(" ") 
    
    currentQueryTermDict = {}
    for element in queryContentElements:
        if element.strip() != "":
            if element.strip() not in currentQueryTermDict:
                currentQueryTermDict[element.strip()] = 1
    currentQueryLength = len(currentQueryTermDict)
    if currentQueryLength not in queryLengthDistributionDict:
        if currentQueryLength == 0:
            print "lineCounter:",lineCounter,"*",currentLine.strip(),"*"
            # print currentQueryTermDict,"*"
            # print "*",currentLine.strip(),"*"
            # print
        queryLengthDistributionDict[currentQueryLength] = 1
    else:
        if currentQueryLength == 0:
            print "lineCounter:",lineCounter,"*",currentLine.strip(),"*"
            # print currentQueryTermDict,"*"
            # print "*",currentLine.strip(),"*"
            # print
        queryLengthDistributionDict[currentQueryLength] += 1
    currentLine = inputFileHandler.readline()     
    lineCounter += 1
    # print lineCounter
    if lineCounter % 1000001 == 0:
        print lineCounter
        break

queryLengthList = []
queryLengthList = queryLengthDistributionDict.keys()
queryLengthList.sort(cmp=None, key=None, reverse=False)
# option1
TOTAL_NUM_OF_QUERIES = 95000
# option2
# TOTAL_NUM_OF_QUERIES = 1000000
totalQueryLength = 0
averageQueryLength = 0
for queryLength in queryLengthDistributionDict:
    print queryLength,queryLengthDistributionDict[queryLength],queryLengthDistributionDict[queryLength]/TOTAL_NUM_OF_QUERIES
    totalQueryLength += queryLengthDistributionDict[queryLength] * queryLength
averageQueryLength = totalQueryLength / TOTAL_NUM_OF_QUERIES
print "total # of queries counted:",lineCounter-1
print "average Query Length:",averageQueryLength
print "inputFileName:",inputFileName
print "Program Ends."


